####################################coding=gbk
import urllib.request
import os
import re
from logutilitie import logutilitie
from publishutilitie import publishutilitie
class warehouse:
    decode = ''
    downPath = os.path.join(os.getcwd(),'down')
    picFileRule = "http://pic(?P<pic_id>\d+)\.bengou.com/comicdata/(?P<file_path>.*?(\.[pP][nN][gG]|\.[jJ][pP][gG]|\.[bB][mM][pP]|\.[gG][iI][fF]|\.[dD][bB]))"
    picLogFileDir = 'pic'
    splitLogFIleForNumberRule = 'split_url_(.*\d).log'
    downUrlList = []
    logNumber = 0
    def __init__(self,decode,logInputDirName='',logNumber=0,clearDir = False):
        self.decode = decode
        self.log = logutilitie(logInputDirName,'',clearDir)
        self.downPath = os.path.join(self.downPath,'bengou/pic/%s' % logInputDirName)
        self.logNumber = logNumber

        
    def run(self,startLogNumber=0):
        self.downPicFormPicLog(startLogNumber)
        
        
    def downFile(self,url):
        #path = self.downFilePath(url)
        #data = urllib.request.urlretrieve(url,path)
        fileName = self.downFilePath(url)
        if(os.path.exists(fileName)):
                return
        try:
            f=urllib.request.urlopen( url )
            sb = f.read(-1)
            f.close()
            
        except   Exception as Err:
            self.log.wirteLog(('down_pic_error_%d.log' % self.logNumber),url)
            return
        try:        
            
            f1= open(fileName,"wb")
            f1.write(sb)
            f1.close()
        except Exception as Err :
            #sERR = str( Err )
            self.log.wirteLog(('down_pic_write_error_%d.log' % self.logNumber),url)
        return 



    def downFilePath(self,url=''):
        m = re.match(self.picFileRule,url)
        picServerId = m.group('pic_id')
        filePath = m.group('file_path')
        savePath = os.path.join(self.downPath,filePath)
        saveDir = os.path.dirname(savePath)
        if (os.path.exists(saveDir) == False):
            os.makedirs(saveDir)
        return savePath
    
    def downPicFormPicLog(self,startLogNumber=0):
        if(type(startLogNumber) != int):
            startLogNumber = 0
        maxLogNumber = startLogNumber + 500
        picDir = os.path.join(self.log.LogFilePath,self.picLogFileDir)
        for parentDir,dirNames,picFileNames in os.walk(picDir):
            
            print('downPicFormPicLog read dir : ', picDir )
            #picFileNames.reverse()
            for fileName in picFileNames :
                logId = re.findall(self.splitLogFIleForNumberRule,fileName);
                if(len(logId)):
                    if(int(logId[0]) > maxLogNumber  or int(logId[0]) < startLogNumber):
                        continue
                else:
                    continue
                print('start file : ',fileName,publishutilitie.getLocationTime())
                asIndexUrlLog = self.log.readLogToList("%s/%s" % tuple([picDir,fileName]))
                asIndexUrlLog = set(asIndexUrlLog)
                picUrlList = []
                for urlInfo in asIndexUrlLog :
                    url = re.match('(.*?)\\n',urlInfo)
                    if(len(url.group(0)) != 0) :
                        self.downFile(url.group(0))
                self.log.removeFile(os.path.join(parentDir,fileName))



if(__name__=='__main__'):
    import configmap
    import sys
    #logInputDirName = input('Enter log dir name --> ')
    logInputDirName = 'free10'
    startLogNumber = int(sys.argv[1])
    logNumber = startLogNumber
    warehouse(configmap.WebDecode,logInputDirName,logNumber).run(startLogNumber)